{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "902e434d-3692-4686-a50c-e7fe0250c84a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import json\n",
    "import time\n",
    "from collections import defaultdict\n",
    "from pytz import timezone\n",
    "from datetime import datetime\n",
    "from sys import getsizeof as getsize\n",
    "import networkx as nx\n",
    "from common_util import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1fe65fdd-26b7-4fb6-9ddf-498836c88135",
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_conversion(var: int):\n",
    "    \"\"\"\n",
    "    二进制单位转换\n",
    "    :param var: 需要计算的变量，bytes值\n",
    "    :return: 单位转换后的变量，kb 或 mb\n",
    "    \"\"\"\n",
    "    assert isinstance(var, int)\n",
    "    if var <= 1024:\n",
    "        return f'占用 {round(var / 1024, 2)} KB内存'\n",
    "    else:\n",
    "        return f'占用 {round(var / (1024 ** 2), 2)} MB内存'\n",
    "\n",
    "def print_json(json_str):\n",
    "    print(json.dumps(json_str, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f501cec9-ba06-4566-9bb1-55b626420515",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['compose-post-service', 'home-timeline-service', 'media-service', 'nginx-web-server', 'post-storage-service', 'social-graph-service', 'text-service', 'unique-id-service', 'url-shorten-service', 'user-mention-service', 'user-service', 'user-timeline-service']\n"
     ]
    }
   ],
   "source": [
    "service_list = ['home-timeline-service', 'post-storage-service', 'media-service', 'nginx-web-server', 'text-service', 'user-mention-service', 'social-graph-service', 'user-service', 'compose-post-service', 'url-shorten-service', 'unique-id-service', 'user-timeline-service']\n",
    "service_list.sort()\n",
    "print(service_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2005f4d-f251-4a84-ad2f-d88989ef55b7",
   "metadata": {},
   "source": [
    "# No Fault"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d3de0cb0-c79a-40c8-96bd-679920fbee61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "service                | #df  ,#null,start,end  \n",
      "compose-post-service   | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "home-timeline-service  | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "media-service          | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "nginx-web-server       | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "post-storage-service   | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "social-graph-service   | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "text-service           | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "unique-id-service      | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "url-shorten-service    | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "user-mention-service   | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "user-service           | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n",
      "user-timeline-service  | 1441,    0,1650421445,1650422885 | 1441,    0,1650497582,1650499022 | 1442,    0,1650480769,1650482210 | 4324,    0\n"
     ]
    }
   ],
   "source": [
    "# 导入原始数据\n",
    "base_dir = \"/home/dyz/trace/dataset/Eadro/sn/no_fault\"\n",
    "files = os.listdir(base_dir)\n",
    "metric_dict = {}\n",
    "for service in service_list:\n",
    "    metric_dict[service] = []\n",
    "\n",
    "for f in files:\n",
    "    metrics_path = os.path.join(base_dir, f, 'metrics')\n",
    "    if not os.path.exists(metrics_path):\n",
    "        continue\n",
    "    metric_files = os.listdir(metrics_path)\n",
    "    for metric_file in metric_files:\n",
    "        if not metric_file.endswith('csv'):\n",
    "            continue\n",
    "        metric_file_path = os.path.join(metrics_path, metric_file)\n",
    "        df = pd.read_csv(metric_file_path, index_col='timestamp').sort_index(ascending=True)\n",
    "        service_name = metric_file[:-4]\n",
    "        metric_dict[service_name].append(df)\n",
    "\n",
    "print('{0:22} | {1:5},{2:5},{3:5},{4:5}'.format('service', '#df', '#null', 'start', 'end'))\n",
    "for k, v in metric_dict.items():\n",
    "    print(f'{k:22} |', end='')\n",
    "    cnt = 0\n",
    "    num_null = 0\n",
    "    for df in v:\n",
    "        cnt += len(df)\n",
    "        n_null = df.isna().sum().sum() + df.isnull().sum().sum()\n",
    "        num_null += n_null\n",
    "        start, end = df.index[[0,-1]]\n",
    "        print(f'{len(df):5},{n_null:5},{start:5},{end:5} |', end='')\n",
    "    print(f'{cnt:5},{num_null:5}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "23c20a0f-f6f4-490f-af71-300445c0c1a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12, 4324, 7)\n"
     ]
    }
   ],
   "source": [
    "# 合并metric\n",
    "t_np_list = []\n",
    "for service in service_list:\n",
    "    df_list = metric_dict[service]\n",
    "    concat_df = pd.concat(df_list, axis=0).sort_index(ascending=True)\n",
    "    # 全都是(4234, 7)\n",
    "    #print(concat_df.shape)\n",
    "    t_np_list.append(concat_df.to_numpy())\n",
    "\n",
    "normal_metrics = np.stack(t_np_list, axis=0)\n",
    "print(normal_metrics.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "51a0ecef-74dd-45c7-a365-3e70f1ce5394",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to ../data/sn/sn_metric_normal.npy, normal metric shape: (12, 4324, 5)\n"
     ]
    }
   ],
   "source": [
    "# 保存normal metric\n",
    "npy_name = '../data/sn/sn_metric_normal.npy'\n",
    "if os.path.exists(npy_name):\n",
    "    os.remove(npy_name)\n",
    "    \n",
    "np.save(npy_name, normal_metrics[:,:,:-2])\n",
    "print(\"saved to {}, normal metric shape: {}\".format(npy_name, normal_metrics[:,:,:-2].shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c66f478-5362-4551-b6aa-afb4548d4f01",
   "metadata": {},
   "source": [
    "# 合并Fault数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "68d886cf-c0c0-4b86-b374-e83184f25ec4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compose-post-service   | 9970,    0\n",
      "home-timeline-service  | 9970,    0\n",
      "media-service          | 9970,    0\n",
      "nginx-web-server       | 9970,    0\n",
      "post-storage-service   | 9970,    0\n",
      "social-graph-service   | 9970,    0\n",
      "text-service           | 9970,    0\n",
      "unique-id-service      | 9970,    0\n",
      "url-shorten-service    | 9970,    0\n",
      "user-mention-service   | 9970,    0\n",
      "user-service           | 9970,    0\n",
      "user-timeline-service  | 9970,    0\n"
     ]
    }
   ],
   "source": [
    "# 导入原始数据\n",
    "base_dir = \"/home/dyz/trace/dataset/Eadro/sn/fault/\"\n",
    "files = os.listdir(base_dir)\n",
    "# metric_dict = {}\n",
    "# for service in service_list:\n",
    "#     metric_dict[service] = []\n",
    "\n",
    "for f in files:\n",
    "    metrics_path = os.path.join(base_dir, f, 'metrics')\n",
    "    if not os.path.exists(metrics_path):\n",
    "        continue\n",
    "    metric_files = os.listdir(metrics_path)\n",
    "    for metric_file in metric_files:\n",
    "        if not metric_file.endswith('csv'):\n",
    "            continue\n",
    "        metric_file_path = os.path.join(metrics_path, metric_file)\n",
    "        df = pd.read_csv(metric_file_path, index_col='timestamp').sort_index(ascending=True)\n",
    "        service_name = metric_file[:-4]\n",
    "        metric_dict[service_name].append(df)\n",
    "\n",
    "#print('{0:22} | {1:5},{2:5},{3:5},{4:5}'.format('service', '#df', '#null', 'start', 'end'))\n",
    "for k, v in metric_dict.items():\n",
    "    print(f'{k:22} |', end='')\n",
    "    cnt = 0\n",
    "    num_null = 0\n",
    "    for df in v:\n",
    "        cnt += len(df)\n",
    "        n_null = df.isna().sum().sum() + df.isnull().sum().sum()\n",
    "        num_null += n_null\n",
    "        start, end = df.index[[0,-1]]\n",
    "        #print(f'{len(df):5},{n_null:5},{start:5},{end:5} |', end='')\n",
    "    print(f'{cnt:5},{num_null:5}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a4c86971-2f37-46f6-8d11-c52d8d61a46b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12, 9970, 7)\n"
     ]
    }
   ],
   "source": [
    "# 合并fault和no_fault的所有metric\n",
    "t_np_list = []\n",
    "for service in service_list:\n",
    "    df_list = metric_dict[service]\n",
    "    concat_df = pd.concat(df_list, axis=0).sort_index(ascending=True)\n",
    "    # 全都是(9970, 7)\n",
    "    # print(concat_df.shape)\n",
    "    t_np_list.append(concat_df.to_numpy())\n",
    "\n",
    "all_metrics = np.stack(t_np_list, axis=0)\n",
    "print(all_metrics.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8c4e61f8-b531-4585-afa3-97280aa6b777",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to ../data/sn/sn_metric_all.npy, metric shape: (12, 9970, 5)\n"
     ]
    }
   ],
   "source": [
    "# 保存所有metric\n",
    "npy_name = '../data/sn/sn_metric_all.npy'\n",
    "if os.path.exists(npy_name):\n",
    "    os.remove(npy_name)\n",
    "    \n",
    "np.save(npy_name, all_metrics[:,:,:-2])\n",
    "print(\"saved to {}, metric shape: {}\".format(npy_name, all_metrics[:,:,:-2].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a1e8fb64-2d9b-46fd-8ec1-41df115e617d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to ../data/sn/sn_time_index.pkl, time index length: 9970\n"
     ]
    }
   ],
   "source": [
    "# 保存所有metric的time index\n",
    "timestamp_index_map = {}\n",
    "for i, tstamp in enumerate(concat_df.index.to_list()):\n",
    "    timestamp_index_map[tstamp] = i\n",
    "    \n",
    "pkl_name = '../data/sn/sn_time_index.pkl'\n",
    "if os.path.exists(pkl_name):\n",
    "    os.remove(pkl_name)\n",
    "\n",
    "save_pkl(pkl_name, timestamp_index_map)\n",
    "print(\"saved to {}, time index length: {}\".format(pkl_name, len(timestamp_index_map)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "trace",
   "language": "python",
   "name": "trace"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
